# -*- coding: utf-8 -*-
# @Time : 2022/8/5 上午10:01
# @Author : WangYunfei
# @FileName: demo4.py
# @Email: yunfei.wang@innvote.com

"""
4. 采集首页新闻列表+详情前10条数据，字段不限，地址：https://36kr.com/。说明：禁止使用自动化等技术

咨迅: https://gateway.36kr.com/api/mis/nav/ifm/subNav/flow

"""
import json
import re

import execjs
import requests

from base.db.db_mongo import OpenMongoDB


class Kr36(object):

    def __init__(self):
        self.db = OpenMongoDB("kr_36", db="kaoshi")
        self.base_url = "https://36kr.com/"

    def get_index_news(self):
        """
        获取首页新闻列表
        """
        res = requests.get(url=self.base_url)
        index_data = re.search(r"<script>window.initialState=(.*?)</script>", res.text)
        index_data = json.loads(index_data.group(1))
        item_list = index_data["homeData"]["data"]["homeFlow"]["data"]["itemList"]
        for item in item_list:
            # 数据类型是 10 代表是文章
            if item["itemType"] == 10:
                detail = self.item_content(item["itemId"])
                _data = item["templateMaterial"]
                _data["widgetContent"] = detail["widgetContent"]
                self.db.db.insert_one(_data)
                print(f"插入Mongodb数据: {_data}")

    def item_content(self, item_id):
        """
        首页文章详情
        """
        detail_url = f"{self.base_url}p/{item_id}"
        res = requests.get(url=detail_url)
        content_data = re.search(r"<script>window.initialState=(.*?)</script>", res.text)
        data = self.js_encrypt_content(json.loads(content_data.group(1)))["articleDetail"]["articleDetailData"]["data"]
        return data

    @staticmethod
    def js_encrypt_content(data):
        """
        解析文章详情
        """
        js = """
        ee = {
            a: require('crypto-js')
        }
        function encrypt_data(initialState) {
            var ne = ee.a.enc.Utf8.parse("efabccee-b754-4c");
            var re, oe = initialState || {};
            oe.isEncrypt && (oe = JSON.parse((re = initialState.state,
                ee.a.AES.decrypt(re, ne, {
                    mode: ee.a.mode.ECB,
                    padding: ee.a.pad.Pkcs7
                }).toString(ee.a.enc.Utf8).toString())));
            return oe
        }
        """
        return execjs.compile(js).call("encrypt_data", data)


if __name__ == '__main__':
    kr = Kr36()
    kr.get_index_news()











